{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python \t\t3.6\n",
      "Tensorflow \t1.0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras \t\t2.0.3\n"
     ]
    }
   ],
   "source": [
    "import sys; print('Python \\t\\t{0[0]}.{0[1]}'.format(sys.version_info))\n",
    "import tensorflow as tf; print('Tensorflow \\t{}'.format(tf.__version__))\n",
    "import keras; print('Keras \\t\\t{}'.format(keras.__version__))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline \n",
    "\n",
    "from sys import stdout\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ../mnist-data/train-images-idx3-ubyte.gz\n",
      "Extracting ../mnist-data/train-labels-idx1-ubyte.gz\n",
      "Extracting ../mnist-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../mnist-data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"../mnist-data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(55000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.train.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2oAAABpCAYAAABLV9A4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUVMXdxvFviSIuGEWWjMiSCEYQF3DUqKjELagIKoor\noKIoCYmI0ShxyeYSxSVqjohGw+sSF9yAqERREnAfIoqAsugQQRRQDBJRJNz3D6aqq2d6Znqm63bf\nnn4+53gsqme6q5+5vVbdX5koihAREREREZHk2KzQAxAREREREZF0+qAmIiIiIiKSMPqgJiIiIiIi\nkjD6oCYiIiIiIpIw+qAmIiIiIiKSMPqgJiIiIiIikjD6oCYiIiIiIpIwOX1QM8b0Nca8b4xZZIy5\nLNSgSpkyDU+ZxkO5hqdMw1Om4SnT8JRpeMo0PGWaf6axG14bY5oBC4AjgaXAm8BpURTNCze80qJM\nw1Om8VCu4SnT8JRpeMo0PGUanjINT5kWxuY5/O5+wKIoij4AMMY8DAwAav2DtW7dOurcuXMON9l0\nzZo1axXQH2UajDINb9asWauiKGpDAx//yrR2yjQ8ZRpeYzMF5VqbyspKVq1aZVCmQem1PzxlGp73\nnFqnXD6otQc+8v69FNi/+g8ZY4YDwwE6duxIRUVFDjfZdBljlqBMg1Km4VVlClnkqkyzo0zDU6bh\nNSTTqp9XrvUoLy+3TWUakF77w1Om4XnPqXWKvZhIFEXjoygqj6KovE2bej84ShaUaXjKNDxlGp4y\nDU+ZxkO5hqdMw1Om4SnTsHL5oLYM6OD9e+eqPmk8ZRqeMo2Hcg1PmYanTMNTpuEp0/CUaXjKtABy\n+aD2JtDVGPM9Y0xz4FRgUphhlSxlGp4yjYdyDU+ZhqdMw1Om4SnT8JRpeMq0ABp9jloURRuMMSOB\nqUAz4N4oiuYGG1kJUqbhKdN4KNfwlGl4yjQ8ZRqeMg1PmYanTAsjl2IiRFH0DPBMoLEIyjQOyjQe\nyjU8ZRqeMg1PmYanTMNTpuEp0/yLvZiIiIiIiIiINExOM2pSGsaOHeva69atA+Cdd95xfRMnTqzx\nOyNGjHDtAw44AIDBgwfHNUQRERERkSZFM2oiIiIiIiIJoxk1qdUpp5wCwGOPPVbnzxljavSNGzfO\ntV944QUADj30UNfXsWPHEEMsWQsWLADgBz/4geu77bbbAPjZz35WkDEl0X//+1/XvuSSS4D0Y9Pb\ncNYd5506dcrT6ERERBpu9erVrv3vf/+71p/zX89uueUWAHr06OH6dt11VwD22muv0EOUQDSjJiIi\nIiIikjD6oCYiIiIiIpIwWvooaexyR6h7yeNuu+3m2n379gXggw8+cH2TJqX2QFy0aBEADzzwgOsb\nM2ZM7oMtYW+99RYAm22W+q6lffv2hRpOYn388ceufffddwPQrFkz11dRUeHakydPBmDkyJF5Gl3y\n/etf/3LtE088EYDKysog1/33v//dtbt16wZAhw4dglx3U2aP0/79+7u+22+/HUgv4uQf503ZihUr\nABg0aJDrO/DAAwEYPny46+vcuXPQ2/3Pf/7j2v/85z+B1GshwBZbbBH09qR0TZkyxbXt43/69Omu\nb+HChbX+rn96hH3u/uabb2r83MaNG3McpcRFM2oiIiIiIiIJoxk1AVIzC08++WSNy/wTT+1MWevW\nrV3ftttuC8D69etd3/777+/ab7/9NgCfffZZwBGXttmzZwOp7CE14yGwcuVKAIYOHVrgkRS3qVOn\nunamb2Fz4c+633vvvQA8/PDDQW+jqfCfO/1ZM8sWEBo2bJjr22qrreIfWIH4hRR23313IH2Gq127\ndkD4WTT/dnr16uX6Vq1aBaTP0Hft2jX4bRfSmjVrALjssstc39y5c4FUwTDQTGJjLF682LX/9Kc/\nATB+/HjXZ7dFAoiiqEHX/f777+c4Oik0zaiJiIiIiIgkjD6oiYiIiIiIJEzRLH2cOHGia9uiADvt\ntJPra9GiBQBnnHGG6/vud78LQJcuXfIxxKK2fPlyIH1a3S559Jc/lZWV1XodY8eOde358+fXuLxf\nv345j7OUzZkzx7Vt8YAhQ4YUajiJY/eRA3jqqacAePPNN7P+/RkzZgDpjwG7t8whhxwSYohFY8OG\nDQA888wzsd2Gv4fdzTffDKTve7fNNtvEdtvFxharAFi2bFmNy0877TQg9TrYVNklhn7hELss9Kc/\n/anrs8+Pcfj9738PwIcffuj67DK1prbc0S8AdsUVVwCZ9+yyyyIBdtxxx/gH1sQsXbrUtW+99dYg\n12kLvvmnrpQqW9DOPn9A+mk+tjCLX5ztggsuAFKFiaBwj2/NqImIiIiIiCRM0cyoXXLJJa5dV3no\ncePGufZ2220HQPfu3YOPx5aRvvTSS12f/w1xsTnuuOOA1DcPAC1btgSgVatWWV3HI4884tp+YREJ\nwz8p2M48+NsplLpRo0a5dmNKkz/xxBNp/wfo2LEjAI8++qjr22effRo7xKLx0ksvAfDKK6+4vl/+\n8pdBb+Pzzz93bVuU4KuvvnJ9pT6j5hdvsbM4tRk8eDAAxphYx1RodrsIvzS5ddVVV8V2u++++65r\n25UjJ5xwgutras/Ddobnoosucn12NiLTMWaL2QDccccdrp3te4emyp/BsTNlvXv3dn12O4fmzZu7\nvu985ztAeqGwtWvXuvaPf/xjIH2mzBZv69mzp+uzxYRK7XnUrjyyRVkg9Zpui4xl47XXXgPSi+PY\nrQ78v+Ef//hHIP1vGJpm1ERERERERBJGH9REREREREQSpt6lj8aYe4F+wIooinpU9bUCHgE6A5XA\noCiKVtd2HSHcc889rm335fKXNM6bNw+At956y/XZ5RF2ChNSS5kynRDr86c77Z5htuCGf512CSRk\nv/TxnHPOYcqUKbRt29ZfUtHMGPM8ecw0k06dOjX4d2688UYAFixYkPFyOy3v760Wh0y5FuJYjcsN\nN9zg2nZ/oLiX2xZDpscccwyQXgTkf//7X1a/6+8HaJeILFmyxPXZggH77ruv69u4cWPjB0tyM/WL\n1Zx66qlAeiGmMWPGBL09fx+1XCU101y88847rm2X/Pk23zz18n300UcHv/2kZLpixQrXfvzxx2tc\nbvfga9OmTfDbtvf7yCOPrHGZv2+lPU2gPknJtD52eWe2e5/6+x8+++yzrm0LkPhLI0MvEUvi+yl7\naoJ/3Nj3rbbQle+AAw5wbfse1t8D0H+/uvPOOwPphS9CS2KmmdjnSH+Zoz0Fx99X0bLZARx88MGu\nbbO272UhdYrD66+/7vrs48EvsmULjtniI3HI5i/9F6Bvtb7LgGlRFHUFplX9W7J01lln8dxzz1Xv\nLkOZ5qSWXHWs5kCZhqdMw1Om4SnT8JRpeHo/FZ4yTZZ6Z9SiKPqnMaZzte4BQJ+q9gRgOhD2TPNq\nDj/88Ixty56U6Vu9etOHfX+Wzc5A1Fe2e8stt3RtewKhLXcKqRPhd9lll3rHXt0hhxySqSDK9mzK\nEvKUaS6mTJni2vYkbv/k93bt2rn29ddfD8DWW28d65hqyTXvx2pI/v3xj1l7TMZ9onBSM/3HP/7h\n2u+99x6QfpJ7XcVE/G++jjrqKNe2J3G/+OKLru+aa66p8ft33nknACNGjGjosIHkZurfV1vUwy/P\n7Z/cngv73On/DXMtgpHUTHPhF7XJJNMsT0hJyfTiiy92bXs89urVy/WdfPLJsd32zJkzAfjkk09c\n39lnnw3AmWee2eDrS0qmmfgrCe67774al9uZA/+1/fnnn6/xc/5Mhp2Zy7RtUihJeT/lF1A7/fTT\ngdQsGqRWJBxxxBF1Xo8/k2bZlWD5kpRMMzn//PNd25bYz1QkxM95jz32AODaa691fZm2Mnn11Vdd\n277O28c7wOzZs4H0Y/gnP/kJAAMHDnR9oWf3Gzt32i6KIrsO8BOgXW0/aIwZboypMMZUNKTiSgna\nXJnGIqtjVZk2iDINT5mGp0zD02t/eMo0PL2fCk+ZFkjOi1yjTSeGRHVcPj6KovIoisrjWEPeFCnT\neNSVqzJtHGUanjINT5mGp9ep8JRpeMo0PGWaX43dR+1TY0xZFEXLjTFlwIp6f6MAdthhBwAOO+yw\nGpdlWj5ZG3sCs11KCbDnnnsCqZPuA9hQDJlaFRUVru0vebT8fWUOPfTQvIypFkVxrNbGXx7mK/CT\nX8Eytcsx/Medv1dNdf6SkZNOOgmAq6++2vVlWo7rF9S56667atyG3Tvx66+/dn0jR44E0osQNVDB\nMp04cSKQfoK0LSLiF1EJxe4J5i937NOnDwDbb799yJtqko99W4zBX8aTR3nP1D9ObLt9+/auL1Rx\ninXr1gHpudoiBf4YbPGSgBJxnNplXQBr1qwBNi2Bs+zx6D/vPfTQQwBcd911rs/fi9UuGR0wYIDr\ns8VGYt5jLW/vp+weZ/5xM3nyZCD9ddruBRz3KSAxyvt7VP9Ys8XU7r77btdnC4i1bdvW9dlTEvy9\nl7M9RcQvnrNhwwYAfvOb37g+u4ddXXs5x6GxM2qTgKFV7aHA02GGU9K+QJnGQcdqeMo0PGUanjIN\nT5mGp0zD0/up8JRpgWRTnv+vbDrRtbUxZilwNXA98KgxZhiwBBgU5yALwS8JbE8W9Mt/2wIajflW\n6LTTTmP69OmsWrWKnXfe2X5iXw4cmfRMjz/+eACmTp1a47KhQ4e6tv3WPJ+q5wq0psiPVb9Et8/O\n6sQtaZl+++23QN2zaJD6JtiW6oX0Uvx18WfU7Ango0ePdn229LL/N+jfvz+QXXGhpGX62GOPAan7\nBY0vlFIb/xtI+y28X17elvFu7Ixk0jLNxSuvvAKkn9jus9/I77333rGOI8mZ+sWsbEEgfzY22+PX\nbuHjt/3tfKxQBUuqZ1r1GEhEpv7KGDuDeNFFF9X4Ob8IwznnnAOkZuUBFi9e7Nr2PZM/ixS6PH+h\n30/Zcvu2aBqkXkNmzJjh+myxqmJQ6Ewt//FpS+f778PtzLpfeGm//fbL6rr9LXw++ugjAIYMGeL6\njj32WCB9JV0mgwcPBoKvBkmTTdXH02q5KPu1g5Lmr3/9a42+c889939RFCnTHFTP1RizKoqiz9Cx\n2mjKNDxlGp4yDU+Zhlc90/LyciorK5VpDvR+Kjxlmizx7ZgnIiIiIiIijdLYYiJNnr/TuV0G6U9t\n2n2sSsHy5ctd2y7L8ZdJ2BNm7fIlCLfnUqmyy578/Wx69uzp2nHvoVSM/MIXNrdslzvWxi5pfPDB\nB13fG2+8kdN1JoG/11GmpV52uXco48ePd21brrl79+6uL1PBp1JV3x6foZelJt2FF17o2naPw48/\n/tj12SIX/pKop5/O7vQZ/3cy7eVnlzIXqHBLXmWaRfnb3/7m2va0h0z84mKZ/PCHP3TtpvbewL4n\n8tnX6qolw9JItqAHZN4b1S6Vf/31112fXYZr91f1bbXVVq49f/78Gm3//YK/d2J1/l6CuS7bz4Zm\n1ERERERERBJGM2rVzJw5E0g/MdTyv6Xr0aNH3sZUaCeeeKJrZyricMYZZwDZFVKQ7EybNg1IP5G1\nb9++ru2f0F2K/BOBLf9btVDsN+4bN26s0eePwZb8f+CBB4KPIQ7+jPjSpUuBTSeQx8UvMGCV0nNo\nQ2SaUfNXc4Se7Uy6ffbZx7XnzJkDpJeSf+6554BU+W5Ilev2C1xlYgsBQGrLHd+BBx4IlMZrm//4\nt+91/GPRzlDYvwHAk08+CaS/TvnHqu33Z9Rt5v6MejHzC6lYdgsCv7S7XZ3hr4yRuvnbaP3oRz8C\n4Pnnn3d9S5YsAeDnP/95nddjC1f5M3SZZJpF22yz1HyWfS982223ub6ysrI6rzMEzaiJiIiIiIgk\njD6oiYiIiIiIJIyWPlbzzDPPALB+/XrXd8QRRwBwwAEHFGRMhTJp0iQA3nrrrRqX9enTx7V/+9vf\n5mtIJePtt9+u0RdqL59iNm7cOCDzicVxmDx5MpD+GLBFB/wx+EtcikHLli1d2+7H5S9p+vzzz4HG\n7RPps4WY7F5tvoMOOiin625K7JJ7SO0z5/P3YCrlAgU77LADkFoG5bf/8Ic/NPj6PvjgA9e2S5r9\n/enGjh3bqHEWI/s+B1LHm7+PZ7du3YDMRVf84lZ+IbZ+/foBsGDBAtdnl43Z5/JiZ4sj+bnYpeX+\n64LdW/aCCy5wffvvvz+Q2scLoEuXLgDsvvvuGW9v7ty5QPr70ab6nOAX/7DLbL/44gvXZ09Revnl\nl13fjjvuCEDHjh1dn/17+O+rsj1V4vzzz3dtW1Qozj3TMtGMmoiIiIiISMLog5qIiIiIiEjCaOkj\nsG7dOte2FaS23HJL12enr+PcJyEpPvvsM9e207z+MlDLXx7S1PZFKRS/4tCMGTMA2G233VzfCSec\nkPcxJc2UKVNiu267hGXevHmur679k/w9V4rtucFfUmKX2vjVy4499lgARo8endX1vfvuu67tV3i0\nVbkyLZfyq2mVOv9519/by9K+ifHwl+3bY9SvHmn3CC0F/jJnu1T5pJNOcn1270X/+LTV9vxlp35F\nYlsl77rrrnN9U6dOBdKfJ4q5quYvfvELAG666aY6f85WCfaXhvrthrKVTSF1KsrDDz/c6OsrFv6y\nw0zV2esyZMgQ18609HG77bZz7ZtvvhmAs846y/Xl65SL6vRKKSIiIiIikjCaUQNuvPFG17ZFA44+\n+mjXZ/dSKQX+t0JvvPFGjcuPP/54QAVE4vCXv/zFtT/99FMg/TiUeF1zzTVA/d9ydu7cGYAJEya4\nPv/E5WLz61//Gkj/ptzOXJ566qlZXYc/8+DPnmXad9E6++yzGzLMJi1TsRX/m+Phw4fnczhNmp+1\n/xi236bbYgSlzBYW8WfZbZEb/7i07wNq29fzyiuvBGD+/Pmuz+7R5r+H8P8OxcbO6gwaNMj12b1l\nv/32W9dn96vMtAdoY9hCTZA6pv29Ka+44oogt9MU2Fny+mYc77zzTtc+/fTTYx1TQ2hGTURERERE\nJGH0QU1ERERERCRhSnbpo1+U4He/+51r2/1D7JR9qbEnUNbGLgtTAZHwbOEFn903SOJxzDHHuPZ7\n772X1e90794dgIMPPjiWMeWb3R/p0UcfdX12Cbh/wn9d/KIDvqFDhwLwwAMP1LjML2hSquxyqEx7\np/l7I+277755G1NT9+yzz2bstwV0evXqlc/hJJq/t5rfzpZ9jJ9yyimuzy59fOmll1xfqH0bC8EW\nmPAfo/6+cda0adOA9OWQdtl5ptNMGsIuW581a1ZO19OU3HPPPa5t97Dzs/fZJaMDBw6Mf2CNoBk1\nERERERGRhCm5GTVbBtmWlQXYsGGDa9tv2P1d3yXF5teQcuR2ltL/HfvNhi3561u9erVr33LLLbVe\nr18q1ZYH3nrrrbMeV9JMnjy5Rl+/fv0KMJLkst8cZjohO9M35eedd55rf/zxx7VeH2QuIZ9JnFsE\nJEXPnj3T/t9Y3//+92u9bM6cOa69xx575HQ7xeqVV14BMpfkHzBgQL6HUxL854ltttnGtW2ZdQnP\nL7QxadIkIL2wwx133AHAVVddld+B5dHhhx9eo2/27NlA+oyafZ/kF1vyX8fse6JMs/CSyvLiiy92\nfV9++WWNn2vZsqVr2yIi/rZcSVLvBzVjTAfg/4B2QASMj6Loj8aYVsAjQGegEhgURdHq2q5HUj76\n6COGDBnCp59+ijHGVfRSpo23dOlSzjvvPFauXKlMA9FxGp4yjUf1XIG2oFxzoUzDq57pmjVrAGWa\nCz2nhqdMkyWbpY8bgIujKOoO/BD4qTGmO3AZMC2Koq7AtKp/SxY233xzbrrpJubNm8drr71mz/tq\ngTJttGbNmnHttdcq04B0nIanTONRPVegrV6ncqNMw6ueadUXi8o0B3pODU+ZJku9M2pRFC0Hlle1\nvzTGzAfaAwOAPlU/NgGYDvwyllHmyF8m1bdvXwA+/PBD19elSxfX9guLxKWsrIyysjJg0/Rrt27d\nWLhwYXOKINM999yzwb9jlz3Y+wypfcLq29ciW+3atQPgoIMOKrpMZ8yYAaQySYokHqcjRowA4NJL\nL61xmS0GAOnLYuvq858bMl1uXXDBBQ0aZ22SmGmc7JK+TEv7Qi53rJ4rsI4ieJ2yS8l9rVu3BmDU\nqFH5Hk6aYs20NuPGjQPgk08+cX32dQPyU0SkeqYtWrTgm2++KdpMs7XZZqk5Afvc/dRTT7k+W1TD\n37dx1113zeq6i/k59aijjgJgzJgxrs+eFjJ+/HjXt3DhQteePn16rdfXvn37IOMq5kztKSR2ttrn\nL3W2S3ABevfuHf/ActCgYiLGmM5AT+B1oF3VhziAT9i0NDLT7ww3xlQYYypWrlyZw1CbpsrKSlth\nbS3KNAhlGp4yDU+ZxqOyshJga7J8nVKm9WtopqBc61NZWclXX30FyjQYPaeGp0wLL+tiIsaYbYHH\ngVFRFK3xT7yPoigyxtT8ynTTZeOB8QDl5eUZfyZufonpioqKGpf7Jel32WWXvIwJYO3atQwcOJBb\nb72VgQMHbvQvK1Smfrly/xuvXPhlv+tiT6L1v32z+vfv79rl5eU1LrffiCQx0/o8+eSTQHpRG1vE\n4dBDD83nUDJKUqYnnngiADfccIPrW7VqVYirdjMZtlw9wN133w2kzwaHkKRM42RfJ7It1JIrmyvw\nUbavU4XMdOrUqTX6OnToAKSKMBVaYzKtuixRx6qdUfPH77/eWX7hAVvYqmPHjkHHYjPt0KEDixcv\nLtpMG2PvvfcG0lcv2UIul19+ueuzW3pku41HMT6n2tcaf/uCRx55pMbP+VsZWJtvnnr7bleT2KJq\noRRLpv5j1n9vUN2ZZ57p2n369IlzSEFlNaNmjNmCTR/SHoyi6Imq7k+NMWVVl5cBK+IZYtP07bff\nMnDgQM444wz35hNlmpMNGzYo08B0nIanTOPh5wp8UdWtXHOgTMPzM/X2yVSmOdBzanjKNDnq/aBm\nNn3V82dgfhRF/m7Ik4ChVe2hwNPhh9c0RVHEsGHD6NatG6NHj/YvUqaNFEUR1157rTINSMdpeMo0\nHso1PGUanjINT5mGp0yTJZuljwcBg4E5xpjZVX1jgOuBR40xw4AlwKBafr9glixZAqRO2PSNHTvW\ntfO9V9XLL7/M/fffzx577OGWAQDfIQGZPvHEE65tp5DXr19f5+/MmzcPqL8wyLBhw1y7U6dONS63\nu8L7S8+yNXPmTJ577jmWLVuWuEwzqTo3Aci8/9fJJ58M1F3gIm5JPE7tceMvD7FLdG+99dacrvtX\nv/oVACNHjszpeuqSxEzj9PXXX9foy3YpU0NkyLW7MeYYEpirLRYAsGjRohqXt2jRAmjYXpVxKKZM\nG8tfPmaX2vl7d/bo0QOACRMmBLm96pkuWLCAppZptoYMGeLad911F5D+/sMW0KivgFkxP6fa50L/\ntcsu45s1a5br8wuNde7cGUjPzxZjCaVYMl27di2Q/p4x0/vVvfbaC8j9PUKhZFP1cSZQ2wkGNXfw\nk3r17t27RhU0Y8x/oij6DGXaKMo0PGUanjKNR/VcjTHzoih6puqfyrURlGl41TMtLy+noqJCmeZA\nz6nhKdNkybqYSDGy39LYmTWfX6QhXye6F5tMJdDr8tBDD8U0kqbH/7Z8++23B2DAgAGu78ILL8z7\nmIrJIYccUqPtz5zb0sa2VC/AcccdB8D555/v+vwXo+7du8cz2BJ23333AaljHOCqq64q1HASwS+W\ntO+++wIwd+5c19e1a9e8j6lU2WJBAPfccw8A5557ruu78sor8z6mUtGmTRvXfuGFF4D0lTbXX389\nUBrvK/xtIqZMmQLA/fff7/peffVV17azZ23bts3P4BLsxRdfBGDZsmV1/pwtGGhXKxSbBpXnFxER\nERERkfjpg5qIiIiIiEjCNLmljzNmzHDtO+64o4AjEamdv/TRX9Ygjde3b9+MbSkcu7Tvoosucn2H\nHXZYoYaTCH6BoGuuuQZIX37fq1evvI+pFNx+++0AXH311a7PX0I9YsQIAL9kPs2bN8/T6Eqb3afu\nyCOPdH2TJk0CUsXKoLSWpw8ePDhjW1LqWprsn7pT7K85mlETERERERFJmCY3ozZz5kzX9ncrt7p0\n6QLAtttum7cxiYiUIr+Yi9S00047AXDvvfcWeCRN38EHHwykChBI8kycONG1bUl1fwuLUppRk/p9\n/vnnNfpskZVRo0blezix0YyaiIiIiIhIwuiDmoiIiIiISMI0uaWPmXg7qzNt2jQAWrVqVajhiIiI\niIhnu+22c+0PP/ywgCORYjB69Oi0/0OqwEhZWVlBxhQHzaiJiIiIiIgkTJObUbv88ssztkVERERE\npPjZbV/87V+aIs2oiYiIiIiIJIw+qImIiIiIiCSMiaIofzdmzErgv8CqvN1ovFoT7r50iqKoTUN/\nSZnWSZluokzDS0qmSwKPpZCUaXgFzxSa3ONfmcaj4Lkq0zop003ynmleP6gBGGMqoigqz+uNxiQp\n9yUp4wghKfclKeMIISn3JSnjCCFJ9yVJY8lFku5HksaSiyTdjySNJRdJuh9JGkuuknJfkjKOEJJy\nX5IyjhAKcV+09FFERERERCRh9EFNREREREQkYQrxQW18AW4zLkm5L0kZRwhJuS9JGUcISbkvSRlH\nCEm6L0kaSy6SdD+SNJZcJOl+JGksuUjS/UjSWHKVlPuSlHGEkJT7kpRxhJD3+5L3c9RERERERESk\nblr6KCIiIiIikjD6oCYiIiIiIpIwef2gZozpa4x53xizyBhzWT5vOxfGmA7GmJeMMfOMMXONMRdW\n9bcyxjxvjFlY9f8dCjA2ZRp+bMo0/NiKMlNIbq7KNJZxKdPw41Km4celTOMZW1HmqkzDS1SmURTl\n5T+gGbAY+D7QHHgb6J6v289x7GVAr6p2S2AB0B24Abisqv8y4A95HpcyVabKtARzVabKVJkqU2Wq\nXJVp0880nzNq+wGLoij6IIqi9cDDwIA83n6jRVG0PIqif1W1vwTmA+3ZNP4JVT82ATg+z0NTpuEp\n0/CKNlPF+aheAAABtElEQVRIbK7KNDxlGp4yDU+ZxqNoc1Wm4SUp03x+UGsPfOT9e2lVX1ExxnQG\negKvA+2iKFpeddEnQLs8D0eZhqdMw2sSmUKiclWm4SnT8JRpeMo0Hk0iV2UaXqEzVTGRBjDGbAs8\nDoyKomiNf1m0aR5Uex00kDINT5nGQ7mGp0zDU6bhKdPwlGl4yjS8JGSazw9qy4AO3r93ruorCsaY\nLdj0x3owiqInqro/NcaUVV1eBqzI87CUaXjKNLyizhQSmasyDU+ZhqdMw1Om8SjqXJVpeEnJNJ8f\n1N4EuhpjvmeMaQ6cCkzK4+03mjHGAH8G5kdRdLN30SRgaFV7KPB0noemTMNTpuEVbaaQ2FyVaXjK\nNDxlGp4yjUfR5qpMw0tUpqGqkmTzH3AMmyqnLAZ+lc/bznHcvdk0vfkOMLvqv2OAHYFpwELgBaBV\nAcamTJWpMi3BXJWpMlWmylSZKldl2rQzNVUDEhERERERkYRQMREREREREZGE0Qc1ERERERGRhNEH\nNRERERERkYTRBzUREREREZGE0Qc1ERERERGRhNEHNRERERERkYTRBzUREREREZGE+X+IUBJIT7+7\nfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x118116400>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15,5))\n",
    "for i in list(range(10)):\n",
    "    plt.subplot(1, 10, i+1)\n",
    "    pixels = mnist.test.images[i]\n",
    "    pixels = pixels.reshape((28, 28))\n",
    "    plt.imshow(pixels, cmap='gray_r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set parameters\n",
    "learning_rate = 0.01\n",
    "training_iteration = 5\n",
    "batch_size = 250\n",
    "print_freq=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1, mean=0.0) #tf.constant(0.0, shape=shape)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TF graph input\n",
    "x = tf.placeholder('float', [None, 784]) # mnist data image of shape 28*28=784\n",
    "y = tf.placeholder('float', [None, 10]) # 0-9 digits recognition => 10 classes\n",
    "\n",
    "#dropout rate = 1 - keep_rate\n",
    "keep_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# keras model for reference\n",
    "\n",
    "# model = Sequential()\n",
    "# model.add(Reshape(input_shape, input_shape=(784,)))\n",
    "\n",
    "# model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(MaxPooling2D((2,2)))\n",
    "\n",
    "# model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(MaxPooling2D((2,2)))\n",
    "\n",
    "# model.add(Flatten())\n",
    "\n",
    "# model.add(Dropout(0.25))\n",
    "# model.add(Dense(256, activation='relu'))\n",
    "\n",
    "# model.add(Dropout(0.25))\n",
    "# model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv2d(name, x, weight, bias):\n",
    "    with tf.name_scope(name) as scope:\n",
    "        #filter definition\n",
    "        f = weight_variable(weight)\n",
    "        b = bias_variable(bias)\n",
    "    \n",
    "        # Construct a dense linear model, with act=relu and dropout\n",
    "        y = tf.nn.relu(tf.nn.conv2d(input=x, filter=f, strides=[1,1,1,1], padding='SAME') +b)\n",
    "        return y  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv2d_maxpool(name, x, d_in, d_out):\n",
    "    # 2 conv layers stride 1,1 and maxpool maxpool strides of 2\n",
    "    with tf.name_scope(name) as scope:\n",
    "        c1 = conv2d(\"conv_1\", x,  [3,3,d_in,d_out],  [d_out])\n",
    "        c2 = conv2d(\"conv_2\", c1, [3,3,d_out,d_out], [d_out])\n",
    "        y = tf.nn.max_pool(value=c2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dropout(name, x, keep_rate):\n",
    "    with tf.name_scope(name) as scope:\n",
    "        return tf.nn.dropout(x, keep_rate)\n",
    "\n",
    "def dense(name, x, weight, bias, activation='linear'):\n",
    "    with tf.name_scope(name) as scope:\n",
    "        # Set model weights\n",
    "        W = weight_variable(weight)\n",
    "        b = bias_variable(bias)\n",
    "\n",
    "        # Construct a dense linear model, with act=relu and dropout\n",
    "        y = tf.matmul(x, W) + b  \n",
    "        \n",
    "        if activation=='relu':\n",
    "            return tf.nn.relu(y)\n",
    "        else:\n",
    "            return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"reshape\") as scope:\n",
    "    layer_0 = tf.reshape(x, [-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_1 = conv2d_maxpool('CONV-3x3x32', layer_0, 1 , 32)\n",
    "layer_2 = conv2d_maxpool('CONV-3x3x64', layer_1, 32, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"flatten\") as scope:\n",
    "    layer_3 = tf.reshape(layer_2, [-1, 7*7*64])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_4  = dropout(\"dropout\", layer_3, keep_rate)\n",
    "layer_5  = dense('dense_256', layer_4, [7*7*64,256], [256], 'relu')\n",
    "\n",
    "layer_6  = dropout(\"dropout\", layer_5, keep_rate)\n",
    "layer_7  = dense('dense_10',  layer_6, [256,10], [10], 'linear')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost functions and Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get the softmax as a separate tensorflow op\n",
    "y_out = tf.nn.softmax(layer_7)\n",
    "\n",
    "# softmax cross entropy descend on y_hat\n",
    "y_hat = layer_7\n",
    "\n",
    "# More name scopes will clean up graph representation\n",
    "with tf.name_scope(\"cost_function\") as scope:\n",
    "    # Minimize error using cross entropy\n",
    "    # Cross entropy\n",
    "    cost_function = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=y_hat))\n",
    "    \n",
    "    # Create a summary to monitor the cost function\n",
    "    tf.summary.scalar(\"cost_function\", cost_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"train\") as scope:\n",
    "    # Gradient descent\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = tf.equal(tf.argmax(y_out, 1), tf.argmax(y, 1))\n",
    "acc = tf.reduce_mean(tf.cast(predictions, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Merge all summaries into a single operator\n",
    "merged_summary_op = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Launch the graph\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# Logs and graph for tensorboard\n",
    "summary_writer = tf.summary.FileWriter('./tensorboard/tf-rewrite', graph=sess.graph)\n",
    "\n",
    "# Init the session\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54750/55000 avg_cost:1.325054 cost:0.184887 acc:0.948000 epoch: 00 acc_test=0.958500028\n",
      "29000/55000 avg_cost:0.194128 cost:0.228156 acc:0.944000"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-a3562ca3840d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0;31m# Fit training using batch data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         loss, accuracy, optm = sess.run([cost_function,acc,optimizer], \n\u001b[0;32m---> 14\u001b[0;31m             feed_dict={x: batch_xs, keep_rate: batch_kr, y: batch_ys})\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mavg_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/local/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    765\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    768\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/local/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/local/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1016\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/opt/local/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1020\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1023\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/local/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1002\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1003\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Training cycle\n",
    "for iteration in range(training_iteration):\n",
    "    total_batch = int(mnist.train.num_examples/batch_size)\n",
    "    # Loop over all batches\n",
    "    avg_loss =0.\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        \n",
    "        # dropout placeholder\n",
    "        batch_kr = 0.75\n",
    "        \n",
    "        # Fit training using batch data\n",
    "        loss, accuracy, optm = sess.run([cost_function,acc,optimizer], \n",
    "            feed_dict={x: batch_xs, keep_rate: batch_kr, y: batch_ys})\n",
    "        \n",
    "        avg_loss += loss\n",
    "        \n",
    "        stdout.write('\\r{}/{} avg_cost:{:6f} cost:{:6f} acc:{:6f}'.format(i*batch_size, \n",
    "                                                                  mnist.train.num_examples, \n",
    "                                                                  avg_loss/(i+1), \n",
    "                                                                  loss, accuracy))\n",
    "        stdout.flush() \n",
    "            \n",
    "    # Display logs per iteration step\n",
    "    if iteration % print_freq ==0 :\n",
    "        accuracy_test = sess.run([acc], feed_dict={x: mnist.test.images, keep_rate: 1.0, y: mnist.test.labels})\n",
    "        print(\" epoch: {:02d} acc_test={:.9f}\".format(iteration, accuracy_test[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Accuracy:\", acc.eval({x: mnist.test.images, keep_rate:1.0, y: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test item #100 is a six\n",
    "pixels = mnist.test.images[100]\n",
    "\n",
    "#predict\n",
    "result = sess.run(y_out, feed_dict={x:[pixels], keep_rate:1.0})\n",
    "dict(zip(range(10), result[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_render(pixels, result, truth):\n",
    "    #pixels, result and truth are np vectors\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.subplot(1, 2, 1)\n",
    "    pixels = pixels.reshape((28, 28))\n",
    "    plt.imshow(pixels, cmap='gray_r')\n",
    "\n",
    "    plt.subplot(1, 2, 2)\n",
    "    \n",
    "    #index, witdh\n",
    "    ind = np.arange(len(result))\n",
    "    width = 0.4\n",
    "\n",
    "    plt.barh(ind,result, width, color='gray')\n",
    "    plt.barh(ind+width,truth,width, color='green')\n",
    "    plt.yticks(ind+width, range(10))\n",
    "    plt.margins(y=0)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "i = random.randint(0,mnist.test.images.shape[0])\n",
    "\n",
    "pixels = mnist.test.images[i]\n",
    "truth  = mnist.test.labels[i]\n",
    "result = sess.run(y_out, feed_dict={x:[pixels], keep_rate:1.0})[0]\n",
    "\n",
    "test_render(pixels, result, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### What went wrong?\n",
    "pixels = mnist.test.images\n",
    "truth = mnist.test.labels\n",
    "\n",
    "feed_dict = {x:pixels,keep_rate:1.0}\n",
    "result = sess.run(y_out, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_correct = result.argmax(axis=1) == truth.argmax(axis=1)\n",
    "incorrect = np.argwhere(index_correct==False).flatten()\n",
    "\n",
    "print(\"Incorrect predictions: {}\".format(len(incorrect)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,5))\n",
    "plt_idx = 1\n",
    "for i in list(incorrect[:16]):\n",
    "    plt.subplot(1, 16, plt_idx)\n",
    "    pixels = mnist.test.images[i]\n",
    "    pixels = pixels.reshape((28, 28))\n",
    "    plt.imshow(pixels, cmap='gray_r')\n",
    "    plt_idx += 1\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = random.choice(list(incorrect))\n",
    "\n",
    "pixels = mnist.test.images[i]\n",
    "truth  = mnist.test.labels[i]\n",
    "\n",
    "feed_dict = {x:[pixels]}\n",
    "feed_dict.update({keep_rate:1.0})\n",
    "result = sess.run(y_out, feed_dict=feed_dict)[0]\n",
    "\n",
    "test_render(pixels, result, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Close the Session when we're done.\n",
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
